﻿/* *****************************************************************************
ShoutcastRunner 0.3

Copyright (c) KEXP 90.3 FM Seattle. All rights reserved. 
www.kexp.org
author: Louis O'Callaghan (louis@kexp.org)
October 2012

Licensed under the Apache License, Version 2.0 (the "License"); you may not use
this file except in compliance with the License. You may obtain a copy of the
License at http://www.apache.org/licenses/LICENSE-2.0  

THIS CODE IS PROVIDED *AS IS* BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
KIND, EITHER EXPRESS OR IMPLIED, INCLUDING WITHOUT LIMITATION ANY IMPLIED
WARRANTIES OR CONDITIONS OF TITLE, FITNESS FOR A PARTICULAR PURPOSE, 
MERCHANTABLITY OR NON-INFRINGEMENT. 

See the Apache Version 2.0 License for specific language governing permissions
and limitations under the License.

SHOUTcast™ is a trademark of AOL LLC.
***************************************************************************** */

using System;
using System.Collections.Generic;
using System.IO;
using System.Linq;
using System.Text;
using log4net;

namespace Kexp.ShoutcastRunner
{
	public static class W3CLogParser
	{
		private static readonly ILog Log = LogManager.GetLogger(typeof(W3CLogParser));

		/// <summary>
		/// This method splits the shoutcast w3c log into daily files.
		/// 
		/// The plan: go through each line in the specified log file. Check the date of each
		/// line against a dictionary of dates and file handles. If the date exists, append the
		/// line to the file handle. If the date doesn't exist, look for an existing file with
		/// the expected output filename. If the file doesn't exist, create it, add to dictionary
		/// and append the line. If the file exists, check its size. If it's small then open it 
		/// for overwriting, add it to the dictionary and append the line. If it's big, then 
		/// skip the line (because presumably that file has already been processed)
		/// 
		/// Also: remove non-sequential comments (comments might appear in 
		/// the middle of the file, appended whenever shoutcast restarts)
		/// </summary>
		/// <param name="filename"></param>
		/// <param name="overwriteSizeLimit">Overwrite existing files below this size. Default is 300K</param>
		/// <returns></returns>
		public static bool ParseDailyLogs(string filename, int overwriteSizeLimit = Settings.DefaultOverwriteW3CSizeLimit)
		{
			var logInfo = new FileInfo(filename);
			if (!logInfo.Exists)
			{
				Log.Error(string.Format("Can't find {0} to parse.", logInfo.Name));
				return false;
			}

			if (!logInfo.IsW3CLog())
			{
				Log.Error("Wrong log type for this parser.");
				return false;
			}

			using (var fileStream = new FileStream(logInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
			{
				using (var streamReader = new StreamReader(fileStream))
				{
					var outputDictionary = new Dictionary<DateTime, TextWriter>();
					var datesToSkip = new List<DateTime>();
					var heading = new StringBuilder();
					DateTime? currentWorkingDate = null;
					DateTime adjustedNow = Helpers.GetAdjustedNowDate();
					int index = -1;
					int previousHeaderLine = -1;

					while (streamReader.Peek() >= 0)
					{
						index++;
						Line line;
						try
						{
							line = new Line(streamReader.ReadLine());
						}
						catch (ArgumentException e)
						{
							Log.Error(e);
							continue;
						}

						// handle the file header.. note that the # comments can occur multiple times in the stream, so ignore any non-sequential comments
						if (line.IsComment())
						{
							if (index - 1 == previousHeaderLine)
							{
								heading.AppendLine(line.ToString(false));
								previousHeaderLine = index;
							}
							continue;
						}

						var lineDate = line.LineDateTime;
						if (lineDate == null ||
							// if this is running just after midnight, don't create a file for today:
							lineDate.Value.Date > adjustedNow.Date || 
							datesToSkip.Any(d => d.Date == lineDate.Value.Date))
							continue;

						// since line dates will be sequential, we should flush the previous file
						// when the date changes
						if (currentWorkingDate == null || currentWorkingDate.Value.Date != lineDate.Value.Date)
						{
							TextWriter previousFile;
							if (currentWorkingDate != null &&
								outputDictionary.TryGetValue(currentWorkingDate.Value.Date, out previousFile))
								previousFile.Flush();

							currentWorkingDate = lineDate.Value.Date;
						}

						var outputFile = outputDictionary.ContainsKey(lineDate.Value.Date)
											? outputDictionary[lineDate.Value.Date]
											: null;

						if (outputFile == null)
						{
							var datedFileName = Helpers.GetDatedFileName(logInfo.Name, lineDate.Value);
							var outputFilePath = Path.Combine(logInfo.DirectoryName, datedFileName);
							var outputFileInfo = new FileInfo(outputFilePath);

							if (outputFileInfo.Exists)
							{
								if (outputFileInfo.Length > overwriteSizeLimit)
								{
									datesToSkip.Add(lineDate.Value.Date);
									continue;
								}
								outputFileInfo.CopyTo(outputFileInfo.Name + DateTime.Now.Ticks);
								outputFileInfo.Delete();
							}

							outputFile = outputFileInfo.CreateText();
							outputDictionary.Add(lineDate.Value.Date, outputFile);

							// add header to new files
							outputFile.Write(heading.ToString());
						}

						outputFile.WriteLine(line.ToString());
					}

					// cleanup
					foreach (var file in outputDictionary)
					{
						TextWriter writer = file.Value;
						if (writer != null)
						{
							writer.Close();
							writer.Dispose();
						}
					}
				}
			}

			return true;
		}

	    /// <summary>
	    /// This method splits the shoutcast w3c log into daily files, but also evaluates and alters the file contents 
	    /// according to criteria established by Triton Media for their audience metrics service:
	    /// 
	    /// - datetimes in UTC (though the daily files themselves will still be named and split according to local times).
	    /// - group listener sessions by IP address. *
        /// - truncate song metadata/request path. **
        /// - report listener sessions by start time rather than stop time. ***
        /// 
	    /// * That is, if a line shows an IP at 12:00:00 with a duration of 30 seconds and then that same IP again at 
	    /// 12:00:30 with a duration of 60 seconds, assume it's the same listener and they should appear in the log only 
	    /// once with a duration of 90 seconds.
	    /// 
	    /// ** It seems to be a function of in-stream artist/song metadata that the same listener will appear so often, 
	    /// with a different request path that references said metadata. It's not needed so remove it.
	    /// 
	    /// *** Each listener session/line should reference the start date/time of the initial connection, rather than 
	    /// the reporting date/time as Shoutcast does it. For example, if shoutcast reports 12:00:00 with a duration of 
	    /// 30 seconds, the line should instead report a start date of 11:59:30 with a 30 second duration.
	    /// </summary>
	    /// <param name="filename"></param>
	    /// <param name="overwriteSizeLimit"></param>
	    /// <returns></returns>
	    public static bool ParseTritonDailyLogs(string filename, int overwriteSizeLimit = Settings.DefaultOverwriteW3CSizeLimit)
		{
			var logInfo = new FileInfo(filename);
			if (!logInfo.Exists)
			{
				Log.Error(string.Format("Can't find {0} to parse.", logInfo.Name));
				return false;
			}

			if (!logInfo.IsW3CLog())
			{
				Log.Error("Wrong log type for this parser.");
				return false;
			}

			using (var fileStream = new FileStream(logInfo.FullName, FileMode.Open, FileAccess.Read, FileShare.ReadWrite))
			{
				using (var streamReader = new StreamReader(fileStream))
				{
					var outputFileDictionary = new Dictionary<DateTime, TextWriter>();
					var datesToSkip = new List<DateTime>();
					var heading = new StringBuilder();
					var listeners = new ListenerDictionary();
					DateTime? currentWorkingDate = null;
					DateTime adjustedNow = Helpers.GetAdjustedNowDate();
					int index = -1;
					int previousHeaderLine = -1;

					// compile listener and output file info
					while (streamReader.Peek() >= 0)
					{
						index++;
						Line line;
						try
						{
							line = new Line(streamReader.ReadLine());
						}
						catch (ArgumentException e)
						{
							Log.Error(e);
							continue;
						}

						// handle the file header.. note that the # comments can occur multiple 
                        // times in the stream, so ignore any non-sequential comments
						if (line.IsComment())
						{
							if (index - 1 == previousHeaderLine)
							{
								heading.AppendLine(line.ToString(false));
								previousHeaderLine = index;
							}
							continue;
						}

						if (line.LineDateTime == null ||
							// if this is running just after midnight, don't create a file for today:
							line.LineDateTime.Value.Date > adjustedNow.Date ||
							datesToSkip.Any(d => d.Date == line.LineDateTime.Value.Date))
							continue;

						listeners.AddLine(line);

						var lineDate = line.LineDateTime.Value;
						if (currentWorkingDate == null || 
							currentWorkingDate.Value.Date != lineDate.Date)
							currentWorkingDate = lineDate.Date;

						if (!outputFileDictionary.ContainsKey(lineDate.Date))
						{
							var datedFileName = Helpers.GetDatedFileName(logInfo.Name, lineDate);
							var outputFilePath = Path.Combine(logInfo.DirectoryName, datedFileName);
							var outputFileInfo = new FileInfo(outputFilePath);

							if (outputFileInfo.Exists)
							{
								if (outputFileInfo.Length > overwriteSizeLimit)
								{
									datesToSkip.Add(lineDate.Date);
									continue;
								}
								outputFileInfo.CopyTo(outputFileInfo.Name + DateTime.Now.Ticks);
								outputFileInfo.Delete();
							}

							TextWriter outputFile = outputFileInfo.CreateText();
							outputFileDictionary.Add(lineDate.Date, outputFile);

							// add header to new files
							outputFile.Write(heading.ToString());
						}
					}

					// write the output and clean up
					foreach (var file in outputFileDictionary)
					{
						DateTime fileDate = file.Key;
						var visits = listeners.GetVisits(fileDate);
						visits.Sort((v1, v2) => v1.StartDate.Value.CompareTo(v2.StartDate.Value));

    					TextWriter writer = file.Value;
						if (writer != null)
						{
							foreach (var visit in visits)
							{
								writer.WriteLine(visit.ToString(true));
							}

							writer.Close();
							writer.Dispose();
						}
					}
				}
			}

			return true;
		}

		public class ListenerDictionary
		{
			private readonly Dictionary<string, Listener> _dictionary;

			public ListenerDictionary()
			{
				_dictionary = new Dictionary<string, Listener>();
			}

			public void AddLine(Line line)
			{
				Listener listener;
				if (_dictionary.TryGetValue(line.ClientIp, out listener))
				{
					listener.AddVisit(new Visit(line));
				}
				else
				{
					_dictionary.Add(line.ClientIp, new Listener(new Visit(line)));
				}
			}

			public List<Visit> GetAllVisits()
			{
				var allVisits = new List<Visit>();
				foreach (var listener in _dictionary.Values)
					allVisits.AddRange(listener.Visits);

				return allVisits;
			}

			public List<Visit> GetVisits(DateTime date)
			{
			    var subset = new List<Visit>();
                foreach (var listener in _dictionary.Values)
                    subset.AddRange(listener.Visits.FindAll(v => v.StartDate != null && v.StartDate.Value.Date == date.Date));

			    return subset;
			}
		}

		public class Listener
		{
			private const int SecondsMargin = 1;

			public Listener()
			{
				Visits = new List<Visit>();
			}

			public Listener(Visit visit):this()
			{
				AddVisit(visit);
			}

			public List<Visit> Visits { get; set; }

			// find out if this visit is connected to a previous visit
			// if yes, augment the previous visit
			// if no, add a new one
			public void AddVisit(Visit visit)
			{
				var previous = GetPreviousConnectedVisit(visit);

				if (previous != null)
				{
				    int accumulated = visit.StopDate != null && previous.StopDate != null
				        ? (int)(visit.StopDate.Value - previous.StopDate.Value).TotalSeconds
				        : visit.Duration;

                    previous.Duration = previous.Duration + accumulated;
                    previous.StopDate = visit.StopDate;
                    previous.BytesSent = previous.BytesSent + visit.BytesSent;
                    previous.Status = visit.Status;
                    //previous.UserAgent = visit.UserAgent;
                }
				else
					Visits.Add(visit);
			}

			public Visit GetPreviousConnectedVisit(Visit visit)
			{
				if (visit.StopDate == null)
					return null;

				var previousDate = visit.StopDate.Value.Subtract(TimeSpan.FromSeconds(visit.Duration));
				var previousDateLowerLimit = previousDate.Subtract(TimeSpan.FromSeconds(SecondsMargin));
				var previousDateUpperLimit = previousDate.AddSeconds(SecondsMargin);

				foreach (var v in Visits)
				{
					if (v.StopDate >= previousDateLowerLimit && 
						v.StopDate <= previousDateUpperLimit)
						return v;
				}

				return null;
			}
		}

		public class Visit
		{
			public string ClientIp { get; set; }
			public string ClientDns { get; set; }
            public DateTime? StartDate { get; set; }
			public DateTime? StopDate { get; set; }	// the line date could be considered "end date"
			public int Duration { get; set; }		// subtract this from the linedate to consider a "start date"
			public string Status { get; set; }
			public string UserAgent { get; set; }
			public long BytesSent { get; set; }

            public Visit() {}

			public Visit(Line line)
			{
				ClientIp = line.ClientIp;
				ClientDns = line.ClientDns;
				StopDate = line.LineDateTime;
				BytesSent = line.BytesSentLong;
				Duration = line.DurationInt;
				Status = line.RequestStatus;
				UserAgent = line.UserAgent;
			    StartDate = GetStartDate();
			}

			public int GetAverageBandwidth()
			{
				return (Duration > 0) ? (int)(BytesSent * 8) / Duration : 0;
			}

			public DateTime? GetStartDate()
			{
				if (StopDate == null)
					return null;

				return StopDate.Value.Subtract(TimeSpan.FromSeconds(Duration));
			}

			public string ToString(bool utc = false)
			{
				if (StartDate == null)
					return null;

				var date = utc ? StartDate.Value.ToUniversalTime() : StartDate.Value;

				return string.Format("{0} {1} {2}-{3,2:D2}-{4,2:D2} {5,2:D2}:{6,2:D2}:{7,2:D2} /stream?title=Unknown {8} {9} {10} {11} {12}",
					ClientIp, ClientDns,
					date.Year, date.Month, date.Day, date.Hour, date.Minute, date.Second,
					Status, UserAgent, BytesSent, Duration, GetAverageBandwidth());
			}
		}

		public class Line
		{
			// #Fields: c-ip c-dns date time cs-uri-stem c-status cs(User-Agent) sc-bytes x-duration avgbandwidth
			// 11.111.11.111 22.222.22.222 2012-09-05 13:05:22 /stream?title=KEXP%2090%2E3%20FM%20%2D%20Time%3A%2012%3A59%20PM%20Artist%3A%20Frank%20Black%20Song%3A%20%60Hang%20On%20to%20Your%20Ego%60%20Album%3A%20Frank%20Black 200 vlc%2F1%2E1%2E11 557056 110 40512
			private readonly string _lineString;

			public string ClientIp { get; private set; }
			public string ClientDns { get; private set; }
			public string DateString { get; private set; }
			public string TimeString { get; private set; }
			public string RequestPath { get; private set; }
			public string RequestStatus { get; private set; }
			public string UserAgent { get; private set; }
			public string BytesSent { get; private set; }
			public string Duration { get; private set; }
			public string AverageBandwidth { get; private set; }

			public DateTime? LineDateTime { get; private set; }
			public long BytesSentLong { get; private set; }
			public int DurationInt { get; private set; }

			public Line(string lineString)
			{
				_lineString = lineString;
				if (IsComment() && !IsFieldLine())
					return;

				// tried regular expressions, this is actually better and faster
				var segments = _lineString.Split(' ');
				int index = 0;
				if (IsFieldLine())
				{
					if (segments.Length < 11)
						throw new ArgumentException("Line string is malformed.");

					index++;
				}
				else
				{
					if (segments.Length < 10)
						throw new ArgumentException("Line string is malformed.");
				}

				ClientIp = segments[index++];
				ClientDns = segments[index++];
				DateString = segments[index++];
				TimeString = segments[index++];
				RequestPath = segments[index++];
				RequestStatus = segments[index++];
				UserAgent = segments[index++];
				BytesSent = segments[index++];
				Duration = segments[index++];
				AverageBandwidth = segments[index];

				LineDateTime = GetDateTime();
				BytesSentLong = GetBytesSent();
				DurationInt = GetDuration();
			}

			public bool IsComment()
			{
				return _lineString.TrimStart()[0] == '#';
			}

			public bool IsFieldLine()
			{
				return _lineString.TrimStart().IndexOf("#Fields:", StringComparison.Ordinal) == 0;
			}

			private long GetBytesSent()
			{
				long bytesSent;
				return long.TryParse(BytesSent, out bytesSent) ? bytesSent : 0;
			}

			private DateTime? GetDateTime()
			{
				int month, day, year, hour, minute, second;
				var dateSegments = DateString.Split('-');
				var timeSegments = TimeString.Split(':');

				if (dateSegments.Length != 3 || timeSegments.Length != 3)
					return null;

				if (!int.TryParse(dateSegments[0], out year) ||
					!int.TryParse(dateSegments[1], out month) ||
					!int.TryParse(dateSegments[2], out day) ||
					!int.TryParse(timeSegments[0], out hour) ||
					!int.TryParse(timeSegments[1], out minute) ||
					!int.TryParse(timeSegments[2], out second))
					return null;

				return new DateTime(year, month, day, hour, minute, second);
			}

			private int GetDuration()
			{
				int seconds;
				return int.TryParse(Duration, out seconds) ? seconds : 0;
			}

            public new string ToString()
            {
                return ToString(false);
            }

			public string ToString(bool truncateRequestPath)
			{
				if (!truncateRequestPath || IsComment())
					return _lineString;

				return string.Format("{0} {1} {2} {3} {4} {5} {6} {7} {8} {9}",
					ClientIp, ClientDns, DateString, TimeString, 
                    TruncateRequestPath(RequestPath), RequestStatus, UserAgent, 
                    BytesSent, Duration, AverageBandwidth);
			}

			public string TruncateRequestPath(string requestPath)
			{
				// let's make this simple since this is all we need
				return "/stream?title=Unknown";
			}
		}
	}
}
